First cut as implementing issue #58 ...

apt/GNUmakefile.in

@@ -31,12 +31,12 @@ include $(OBJDIR)/Makeconf
 SUBDIRS		= 
 
 BIN_SCRIPTS	= manage_profile manage_instance manage_dataset \
-                  create_instance rungenilib
+                  create_instance rungenilib ns2rspec nsgenilib.py
 SBIN_SCRIPTS	= apt_daemon aptevent_daemon portal_xmlrpc
 LIB_SCRIPTS     = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm \
 		  APT_Aggregate.pm APT_Utility.pm
 WEB_BIN_SCRIPTS = webmanage_profile webmanage_instance webmanage_dataset \
-                  webcreate_instance webrungenilib 
+                  webcreate_instance webrungenilib webns2rspec
 WEB_SBIN_SCRIPTS= webportal_xmlrpc
 LIBEXEC_SCRIPTS	= $(WEB_BIN_SCRIPTS) $(WEB_SBIN_SCRIPTS)
 USERLIBEXEC     = rungenilib.proxy genilib-jail genilib-iocage

apt/manage_profile.in

@@ -334,9 +334,9 @@ if (defined($rspec)) {
 
 #
 # See if this is a Parameterized Profile. Generate and store the form
-# data if it is.
+# data if it is. Only python scripts of course.
 #
-if (defined($script) && $script ne "") {
+if (defined($script) && $script ne "" && $script =~ /^import/m) {
     my ($fh, $filename) = tempfile();
     fatal("Could not open temporary file for script")
 	if (!defined($fh));

apt/ns2rspec.in

+#!/usr/bin/perl -w
+
+#
+# Copyright (c) 2000-2016 University of Utah and the Flux Group.
+# 
+# {{{EMULAB-LICENSE
+# 
+# This file is part of the Emulab network testbed software.
+# 
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+# 
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+# License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# }}}
+#
+use strict;
+use English;
+use Getopt::Std;
+use File::Temp qw(tempfile :mktemp :POSIX );
+use POSIX qw(:signal_h);
+use POSIX ":sys_wait_h";
+use File::stat;
+
+#
+# Convert an NS file into rspec using geni-lib and some lxml parsing.
+#
+sub usage()
+{
+    print STDOUT "Usage: ns2rspec [options] nsfile\n";
+ 
+    exit(-1);
+}
+my $optlist    = "do:";
+my $debug      = 0;
+my $ofile;
+
+#
+# Configure variables
+#
+my $TB       = "@prefix@";
+my $TBOPS    = "@TBOPSEMAIL@";
+my $CONTROL  = "@USERNODE@";
+my $PARSENS  = "$TB/libexec/parse-ns";
+my $NSGENILIB= "$TB/bin/nsgenilib.py";
+
+# Locals
+my $this_user;
+my $nsfile;
+
+# Protos
+sub fatal($);
+
+#
+# Turn off line buffering on output
+#
+$| = 1;
+
+#
+# Untaint the path
+# 
+$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
+delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
+
+if ($UID == 0) {
+    die("Please do not run this as root!");
+}
+
+#
+# Testbed Support libraries
+#
+use lib "@prefix@/lib";
+use libdb;
+use libtestbed;
+use User;
+
+#
+# Parse command arguments. Once we return from getopts, all that should
+# left are the required arguments.
+#
+my %options = ();
+if (! getopts($optlist, \%options)) {
+    usage();
+}
+if (defined($options{"d"})) {
+    $debug = 1;
+}
+if (defined($options{"o"})) {
+    $ofile = $options{"o"};
+}
+if (@ARGV != 1) {
+    usage();
+}
+$nsfile = $ARGV[0];
+
+#
+# Must taint check!
+#
+if ($nsfile =~ /^([-\w\/\.]+)$/) {
+    $nsfile = $1;
+}
+else {
+    die("Bad data in argument: $nsfile.");
+}
+
+my $tmpdir   = mktemp("/tmp/ns2rspec.XXXXXX");
+my $irfile   = "irfile.xml";
+my $xmlfile  = "rspec.xml";
+
+if (! mkdir("$tmpdir", 0755)) {
+    fatal("Could not create temp directory");
+}
+if (! chdir($tmpdir)) {
+    die("Could not chdir to $tmpdir: $!\n");
+}
+
+#
+# Use the NS parser in anon mode, to generate the intermediate
+# representation.
+#
+open ERR, "$PARSENS -n -p -a -c $nsfile 2>&1 > $irfile |";
+
+#
+# Now read in the results from stderr.
+#
+my $errs = "";
+while (<ERR>) {
+    $errs .= $_;
+}
+close(ERR);
+if ($?) {
+    my $tmp = $?;
+    print STDERR $errs;
+    system("/bin/rm -rf $tmpdir");
+    # This error is shown to the user.
+    if (defined($ofile)) {
+	if (open(OFILE, "> $ofile")) {
+	    print OFILE $errs;
+	    close(OFILE);
+	}
+    }
+    exit($tmp >> 8);
+}
+
+#
+# Ick, first line is a tag.
+#
+open(IR, $irfile)
+    or fatal("Could not open $irfile");
+open(XML, ">$xmlfile")
+    or fatal("Could not open $xmlfile");
+while (<IR>) {
+    next
+	if ($_ =~ /^#/);
+    print XML $_;
+}
+close(IR);
+close(XML);
+system("/bin/mv $xmlfile $irfile") == 0 or
+    fatal("Could not rename irfile");
+
+#
+# The next thing is to feed the IR output of the NS parser into
+# our geni-lib converter. 
+#
+open ERR, "$NSGENILIB $irfile 2>&1 > $xmlfile |";
+
+$errs = "";
+while (<ERR>) {
+    $errs .= $_;
+}
+close(ERR);
+if ($?) {
+    my $tmp = $?;
+    print STDERR $errs;
+    system("/bin/rm -rf $tmpdir");
+    # This error is shown to the user.
+    if (defined($ofile)) {
+	if (open(OFILE, "> $ofile")) {
+	    print OFILE $errs;
+	    close(OFILE);
+	}
+    }
+    exit($tmp >> 8);
+}
+
+if (defined($ofile)) {
+    system("cat $xmlfile > $ofile");
+}
+else {
+    system("cat $xmlfile");
+}
+system("/bin/rm -rf $tmpdir");
+exit(0);
+
+sub fatal($) {
+    my ($mesg) = $_[0];
+
+    print STDERR "*** $0:\n".
+	         "    $mesg\n";
+
+    system("/bin/rm -rf $tmpdir")
+	if (defined($tmpdir));
+    exit(-1);
+}

apt/nsgenilib.py.in

+#!/usr/local/bin/python
+#
+# Copyright (c) 2005-2016 University of Utah and the Flux Group.
+# 
+# {{{EMULAB-LICENSE
+# 
+# This file is part of the Emulab network testbed software.
+# 
+# This file is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+# 
+# This file is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
+# License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+# 
+# }}}
+#
+import sys
+import getopt
+import os, os.path
+import pwd
+import traceback
+import string
+import socket
+import re
+import HTMLParser
+
+# Configure stuff.
+OURDOMAIN = "@OURDOMAIN@";
+
+# Testbed specific stuff
+sys.path.append("/usr/local/lib/geni-lib")
+
+# Geni lib stuff.
+import geni.portal as portal
+import geni.rspec.pg as RSpec
+import geni.rspec.igext as IG
+import geni.urn as URN
+import geni.namespaces as GNS
+
+pc = portal.Context() 
+rspec = RSpec.Request()
+tour = IG.Tour()
+
+# This is how we read the NS parser output XML.
+from lxml import etree
+
+def Fatal(message):
+    print >> sys.stderr, message
+    sys.exit(1)
+
+def Usage():
+    print "usage: " + sys.argv[0] + " [option...] irfile"
+    sys.exit(-1);
+    pass
+
+if len(sys.argv) < 2:
+    Usage();
+    pass
+
+NSfile = sys.argv[1];
+
+try:
+    tree = etree.parse(NSfile);
+except err:
+    Fatal("Could not parse IR file: " + str(err))
+    pass
+
+#
+# First find the nodes and links. Do the nodes first so we build the interfaces
+# we need for the links.
+#
+nodes  = {};
+lans   = {};
+ifaces = {};
+
+for child in tree.getroot():
+    if child.tag == "virt_nodes":
+        row = child.find("row")
+        vname = row.find("vname").text
+        # We might end up changing this later, if we determine its a VM.
+        node = RSpec.RawPC(vname)
+        for element in row:
+            #
+            # We handle a subset of node things.
+            #
+            if element.tag == "type":
+                node.hardware_type = element.text
+                if element.text == "pcvm" or re.match(r".*\-vm$", element.text):
+                    node.type = RSpec.NodeType.XEN
+                    pass
+            elif element.tag == "osname":
+                #
+                # Convert NS project/osname to rspec project//osname.
+                # But if no project, add emulab-ops (clearly wrong).
+                osname = element.text
+                if osname.find("/") < 0:
+                    osname = "emulab-ops//" + osname
+                elif osname.find("//") < 0:
+                    osname = osname.replace("/", "//");
+                    pass
+                node.disk_image = "urn:publicid:IDN+" + OURDOMAIN + "+image+" + osname
+            elif element.tag == "fixed":
+                node.component_id = URN.Node(OURDOMAIN, element.text)
+            elif element.tag == "ips":
+                ips = element.text.split()
+                for token in ips:
+                    vport,ip = token.split(":")
+                    iface = node.addInterface("eth" + vport);
+                    iface.addAddress(RSpec.IPv4Address(ip, "255.255.255.0"))
+                    # This is the "member" field in virt_lan.
+                    ifaces[vname + ":" + vport] = iface
+                    pass
+            elif element.tag == "tarfiles" and element.text:
+                tarfiles = element.text.split(";")
+                for token in tarfiles:
+                    directory,filename = token.split()
+                    node.addService(RSpec.Install(filename,directory))
+                    pass
+                pass
+            elif element.tag == "failureaction" and element.text == "nonfatal":
+                raw = etree.Element("{%s}failure_action" %
+                                    (RSpec.Namespaces.EMULAB.name))
+                raw.attrib["action"] = "nonfatal"
+                node.addRawElement(raw)
+                pass
+            pass
+        nodes[vname] = node
+        rspec.addResource(node)
+        pass
+    if child.tag == "virt_lan_lans":
+        row = child.find("row")
+        vname = row.find("vname").text
+        lan = RSpec.LAN(vname);
+        lans[vname] = lan;
+        rspec.addResource(lan)
+        pass
+    pass
+
+#
+# Now we can do the virt_lans, with the links and interfaces we created
+# above.
+#
+for child in tree.getroot():
+    if child.tag == "virt_lans":
+        row = child.find("row")
+        vname  = row.find("vname").text
+        member = row.find("member").text
+        lan    = lans[vname]
+        iface  = ifaces[member]
+        mask   = row.find("member").text
+        lan.addInterface(iface)
+        #
+        # A lot of these things are per virt_lan, but they are really
+        # for the entire lan. 
+        #
+        mask = row.find("member").text
+        iface.netmask = mask;
+
+        if row.find("trivial_ok") != None:
+            trivial_ok = int(row.find("trivial_ok").text)
+            if trivial_ok:
+                lan.trivial_ok = True
+                pass
+            pass
+        if row.find("encap_style") != None:
+            encap_style = row.find("encap_style").text
+            if encap_style == "vlan":
+                lan.vlan_tagging = True
+                pass
+            pass
+        if row.find("emulated") != None:
+            emulated = int(row.find("emulated").text)
+            if emulated:
+                lan.link_multiplexing = True;
+                pass
+            pass
+        pass
+    pass
+
+#
+# Other various things that are in the NS file, that we can handle.
+#
+for child in tree.getroot():
+    if child.tag == "portal":
+        row = child.find("row")
+        for element in row:
+            if element.tag == "description":
+                tour.Description(tour.TEXT, element.text)
+            elif element.tag == "instructions":
+                tour.Instructions(tour.TEXT, element.text)
+                pass
+            pass
+        rspec.addTour(tour)
+        pass
+    if child.tag == "experiments":
+        row = child.find("row")
+        for element in row:
+            if row.find("encap_style") != None:
+                encap_style = row.find("encap_style").text
+                if encap_style == "vlan":
+                    for name,lan in lans.iteritems():
+                        lan.vlan_tagging = True
+                        pass
+                    pass
+                pass
+            if row.find("multiplex_factor") != None:
+                factor = int(row.find("multiplex_factor").text)
+                rspec.setCollocateFactor(factor)
+                pass
+            pass
+        pass
+    #
+    # We only do the startup command right now, since there is no
+    # event mechanism.
+    #
+    if child.tag == "virt_programs":
+        row = child.find("row")
+        vnode = row.find("vnode").text
+        vname = row.find("vname").text
+        cmd   = row.find("command").text
+        if vname == vnode + "_startcmd":
+            foo = re.match(r"^\((.*) ; /usr/local/etc/emulab.*\)", cmd);
+            if foo:
+                parser = HTMLParser.HTMLParser()
+                cmd = parser.unescape(foo.group(1));
+                nodes[vnode].addService(RSpec.Execute("sh", cmd))
+            pass
+        pass
+    #
+    # Watch for desires that specify a shared node.
+    #
+    if child.tag == "virt_node_desires":
+        row = child.find("row")
+        vname  = row.find("vname").text
+        desire = row.find("desire").text
+        if desire == "pcshared":
+            nodes[vname].exclusive = False
+            pass
+        pass
+    pass
+
+pc.printRequestRSpec(rspec)

tbsetup/ns2ir/parse-ns.in

 #!/usr/bin/perl -wT
 
 #
-# Copyright (c) 2000-2014 University of Utah and the Flux Group.
+# Copyright (c) 2000-2016 University of Utah and the Flux Group.
 # 
 # {{{EMULAB-LICENSE
 # 
@@ -40,16 +40,17 @@ use POSIX ":sys_wait_h";
 sub usage()
 {
     print STDOUT
-	"Usage: parse-ns [options] pid gid eid nsfile\n".
+	"Usage: parse-ns [options] [pid gid eid] nsfile\n".
         "Where options and arguments are those required by parse.tcl\n";
  
     exit(-1);
 }
-my $optlist  = "nqap";
+my $optlist  = "nqapc";
 my $anonmode = 0;
 my $passmode = 0;
 my $impotent = 0;
 my $debug    = 0;
+my $catout   = 0;
 
 #
 # Configure variables
@@ -136,6 +137,9 @@ if (defined($options{"n"})) {
 if (defined($options{"p"})) {
     $passmode = 1;
 }
+if (defined($options{"c"})) {
+    $catout = 1;
+}
 if (@ARGV != 4 && @ARGV != 1) {
     usage();
 }
@@ -342,8 +346,14 @@ if ($?) {
 	   error => ['ns_parse_failed']},
 	  "Parsing failed (error code $exit_status)!");
 }
-exit(0)
-    if ($impotent);
+if ($catout) {
+    system("/bin/cat $outfile");
+}
+if ($impotent) {
+    unlink($infile);
+    unlink($outfile);
+    exit(0);
+}
 
 #
 # Run the XML converter on the output. 

tbsetup/ns2ir/sim.tcl.in

@@ -133,6 +133,9 @@ Simulator instproc init {args} {
     $self instvar new_node_config;
     array set new_node_config {}
     $self node-config
+
+    $self set description ""
+    $self set instructions ""
 }
 
 # renaming the simulator instance
@@ -485,6 +488,8 @@ Simulator instproc run {} {
     $self instvar parameter_descriptions
     $self instvar simulated
     $self instvar nseconfig
+    $self instvar description
+    $self instvar instructions
     var_import ::GLOBALS::pid
     var_import ::GLOBALS::eid
     var_import ::GLOBALS::errors
@@ -662,6 +667,10 @@ Simulator instproc run {} {
     
     $self spitxml_init
 
+    if { $description != "" || $instructions != "" } {
+	$self spitxml_data "portal" [list "description" "instructions" ] [list $description $instructions ]
+    }
+
     # update the global nseconfigs using a bogus vname
     # i.e. instead of the node on which nse is gonna run
     # which was the original vname field, we just put $ns
@@ -2159,6 +2168,21 @@ Simulator instproc make-simulated {args} {
     set simulated 0
 }
 
+#
+# Portal Stuff
+#
+Simulator instproc description {text} {
+    $self instvar description
+
+    set description $text
+}
+
+Simulator instproc instructions {text} {
+    $self instvar instructions
+
+    set instructions $text
+}
+
 #
 # Spit out XML
 #

tbsetup/ns2ir/tb_compat.tcl.in

@@ -811,34 +811,32 @@ proc tb-proc-tarfiles {cmd args0} { ; # args has special meaning that we
 	    return
 	}
 
-	# Skip the rest in passmode.
-	if {${GLOBALS::anonymous} || ${GLOBALS::passmode}} {
-	    return
-	}
-
-	# Check the tar file to make sure it exists, is readable, etc...
-	if {[string match "*://*" $tarfile]} {
-	    # It is a URL, check for a valid protocol.
-	    if {![::TBCOMPAT::verify-url $tarfile]} {
-		perror "\[$cmd] '$tarfile' is not an http, https, or ftp URL."
+	# Skip verification in passmode.
+	if { !${GLOBALS::anonymous} && !${GLOBALS::passmode}} {
+	    # Check the tar file to make sure it exists, is readable, etc...
+	    if {[string match "*://*" $tarfile]} {
+		# It is a URL, check for a valid protocol.
+		if {![::TBCOMPAT::verify-url $tarfile]} {
+		    perror "\[$cmd] '$tarfile' is not an http, https, or ftp URL."
+		    return
+		}
+	    } elseif {![string match "${::TBCOMPAT::PROJROOT}/*" $tarfile] &&
+		      ![string match "${::TBCOMPAT::GROUPROOT}/*" $tarfile] &&
+		      ![string match "${::TBCOMPAT::USERROOT}/*" $tarfile] &&
+		      (${::TBCOMPAT::SCRATCHROOT} == "" ||
+		       ![string match "${::TBCOMPAT::SCRATCHROOT}/*" $tarfile])} {
+		perror "\[$cmd] '$tarfile' is not in an allowed directory"
+		return
+	    } elseif {![file exists $tarfile]} {
+		perror "\[$cmd] '$tarfile' does not exist."
+		return
+	    } elseif {![file isfile $tarfile]} {
+		perror "\[$cmd] '$tarfile' is not a file."
+		return
+	    } elseif {![file readable $tarfile]} {
+		perror "\[$cmd] '$tarfile' is not readable."
 		return
 	    }
-	} elseif {![string match "${::TBCOMPAT::PROJROOT}/*" $tarfile] &&
-	          ![string match "${::TBCOMPAT::GROUPROOT}/*" $tarfile] &&
-	          ![string match "${::TBCOMPAT::USERROOT}/*" $tarfile] &&
-	          (${::TBCOMPAT::SCRATCHROOT} == "" ||
-	           ![string match "${::TBCOMPAT::SCRATCHROOT}/*" $tarfile])} {
-	    perror "\[$cmd] '$tarfile' is not in an allowed directory"
-	    return
-	} elseif {![file exists $tarfile]} {
-	    perror "\[$cmd] '$tarfile' does not exist."
-	    return
-	} elseif {![file isfile $tarfile]} {
-	    perror "\[$cmd] '$tarfile' is not a file."
-	    return
-	} elseif {![file readable $tarfile]} {
-	    perror "\[$cmd] '$tarfile' is not readable."
-	    return
 	}