Commit 3c7d6120 authored by Leigh Stoller's avatar Leigh Stoller

First cut as implementing issue #58 ...

parent caf91904
......@@ -31,12 +31,12 @@ include $(OBJDIR)/Makeconf
SUBDIRS =
BIN_SCRIPTS = manage_profile manage_instance manage_dataset \
create_instance rungenilib
create_instance rungenilib ns2rspec nsgenilib.py
SBIN_SCRIPTS = apt_daemon aptevent_daemon portal_xmlrpc
LIB_SCRIPTS = APT_Profile.pm APT_Instance.pm APT_Dataset.pm APT_Geni.pm \
APT_Aggregate.pm APT_Utility.pm
WEB_BIN_SCRIPTS = webmanage_profile webmanage_instance webmanage_dataset \
webcreate_instance webrungenilib
webcreate_instance webrungenilib webns2rspec
WEB_SBIN_SCRIPTS= webportal_xmlrpc
LIBEXEC_SCRIPTS = $(WEB_BIN_SCRIPTS) $(WEB_SBIN_SCRIPTS)
USERLIBEXEC = rungenilib.proxy genilib-jail genilib-iocage
......
......@@ -334,9 +334,9 @@ if (defined($rspec)) {
#
# See if this is a Parameterized Profile. Generate and store the form
# data if it is.
# data if it is. Only python scripts of course.
#
if (defined($script) && $script ne "") {
if (defined($script) && $script ne "" && $script =~ /^import/m) {
my ($fh, $filename) = tempfile();
fatal("Could not open temporary file for script")
if (!defined($fh));
......
#!/usr/bin/perl -w
#
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
use strict;
use English;
use Getopt::Std;
use File::Temp qw(tempfile :mktemp :POSIX );
use POSIX qw(:signal_h);
use POSIX ":sys_wait_h";
use File::stat;
#
# Convert an NS file into rspec using geni-lib and some lxml parsing.
#
sub usage()
{
print STDOUT "Usage: ns2rspec [options] nsfile\n";
exit(-1);
}
my $optlist = "do:";
my $debug = 0;
my $ofile;
#
# Configure variables
#
my $TB = "@prefix@";
my $TBOPS = "@TBOPSEMAIL@";
my $CONTROL = "@USERNODE@";
my $PARSENS = "$TB/libexec/parse-ns";
my $NSGENILIB= "$TB/bin/nsgenilib.py";
# Locals
my $this_user;
my $nsfile;
# Protos
sub fatal($);
#
# Turn off line buffering on output
#
$| = 1;
#
# Untaint the path
#
$ENV{'PATH'} = "$TB/bin:$TB/sbin:/bin:/usr/bin:/sbin:/usr/sbin";
delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'};
if ($UID == 0) {
die("Please do not run this as root!");
}
#
# Testbed Support libraries
#
use lib "@prefix@/lib";
use libdb;
use libtestbed;
use User;
#
# Parse command arguments. Once we return from getopts, all that should
# left are the required arguments.
#
my %options = ();
if (! getopts($optlist, \%options)) {
usage();
}
if (defined($options{"d"})) {
$debug = 1;
}
if (defined($options{"o"})) {
$ofile = $options{"o"};
}
if (@ARGV != 1) {
usage();
}
$nsfile = $ARGV[0];
#
# Must taint check!
#
if ($nsfile =~ /^([-\w\/\.]+)$/) {
$nsfile = $1;
}
else {
die("Bad data in argument: $nsfile.");
}
my $tmpdir = mktemp("/tmp/ns2rspec.XXXXXX");
my $irfile = "irfile.xml";
my $xmlfile = "rspec.xml";
if (! mkdir("$tmpdir", 0755)) {
fatal("Could not create temp directory");
}
if (! chdir($tmpdir)) {
die("Could not chdir to $tmpdir: $!\n");
}
#
# Use the NS parser in anon mode, to generate the intermediate
# representation.
#
open ERR, "$PARSENS -n -p -a -c $nsfile 2>&1 > $irfile |";
#
# Now read in the results from stderr.
#
my $errs = "";
while (<ERR>) {
$errs .= $_;
}
close(ERR);
if ($?) {
my $tmp = $?;
print STDERR $errs;
system("/bin/rm -rf $tmpdir");
# This error is shown to the user.
if (defined($ofile)) {
if (open(OFILE, "> $ofile")) {
print OFILE $errs;
close(OFILE);
}
}
exit($tmp >> 8);
}
#
# Ick, first line is a tag.
#
open(IR, $irfile)
or fatal("Could not open $irfile");
open(XML, ">$xmlfile")
or fatal("Could not open $xmlfile");
while (<IR>) {
next
if ($_ =~ /^#/);
print XML $_;
}
close(IR);
close(XML);
system("/bin/mv $xmlfile $irfile") == 0 or
fatal("Could not rename irfile");
#
# The next thing is to feed the IR output of the NS parser into
# our geni-lib converter.
#
open ERR, "$NSGENILIB $irfile 2>&1 > $xmlfile |";
$errs = "";
while (<ERR>) {
$errs .= $_;
}
close(ERR);
if ($?) {
my $tmp = $?;
print STDERR $errs;
system("/bin/rm -rf $tmpdir");
# This error is shown to the user.
if (defined($ofile)) {
if (open(OFILE, "> $ofile")) {
print OFILE $errs;
close(OFILE);
}
}
exit($tmp >> 8);
}
if (defined($ofile)) {
system("cat $xmlfile > $ofile");
}
else {
system("cat $xmlfile");
}
system("/bin/rm -rf $tmpdir");
exit(0);
sub fatal($) {
my ($mesg) = $_[0];
print STDERR "*** $0:\n".
" $mesg\n";
system("/bin/rm -rf $tmpdir")
if (defined($tmpdir));
exit(-1);
}
#!/usr/local/bin/python
#
# Copyright (c) 2005-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
# This file is part of the Emulab network testbed software.
#
# This file is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This file is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this file. If not, see <http://www.gnu.org/licenses/>.
#
# }}}
#
import sys
import getopt
import os, os.path
import pwd
import traceback
import string
import socket
import re
import HTMLParser
# Configure stuff.
OURDOMAIN = "@OURDOMAIN@";
# Testbed specific stuff
sys.path.append("/usr/local/lib/geni-lib")
# Geni lib stuff.
import geni.portal as portal
import geni.rspec.pg as RSpec
import geni.rspec.igext as IG
import geni.urn as URN
import geni.namespaces as GNS
pc = portal.Context()
rspec = RSpec.Request()
tour = IG.Tour()
# This is how we read the NS parser output XML.
from lxml import etree
def Fatal(message):
print >> sys.stderr, message
sys.exit(1)
def Usage():
print "usage: " + sys.argv[0] + " [option...] irfile"
sys.exit(-1);
pass
if len(sys.argv) < 2:
Usage();
pass
NSfile = sys.argv[1];
try:
tree = etree.parse(NSfile);
except err:
Fatal("Could not parse IR file: " + str(err))
pass
#
# First find the nodes and links. Do the nodes first so we build the interfaces
# we need for the links.
#
nodes = {};
lans = {};
ifaces = {};
for child in tree.getroot():
if child.tag == "virt_nodes":
row = child.find("row")
vname = row.find("vname").text
# We might end up changing this later, if we determine its a VM.
node = RSpec.RawPC(vname)
for element in row:
#
# We handle a subset of node things.
#
if element.tag == "type":
node.hardware_type = element.text
if element.text == "pcvm" or re.match(r".*\-vm$", element.text):
node.type = RSpec.NodeType.XEN
pass
elif element.tag == "osname":
#
# Convert NS project/osname to rspec project//osname.
# But if no project, add emulab-ops (clearly wrong).
osname = element.text
if osname.find("/") < 0:
osname = "emulab-ops//" + osname
elif osname.find("//") < 0:
osname = osname.replace("/", "//");
pass
node.disk_image = "urn:publicid:IDN+" + OURDOMAIN + "+image+" + osname
elif element.tag == "fixed":
node.component_id = URN.Node(OURDOMAIN, element.text)
elif element.tag == "ips":
ips = element.text.split()
for token in ips:
vport,ip = token.split(":")
iface = node.addInterface("eth" + vport);
iface.addAddress(RSpec.IPv4Address(ip, "255.255.255.0"))
# This is the "member" field in virt_lan.
ifaces[vname + ":" + vport] = iface
pass
elif element.tag == "tarfiles" and element.text:
tarfiles = element.text.split(";")
for token in tarfiles:
directory,filename = token.split()
node.addService(RSpec.Install(filename,directory))
pass
pass
elif element.tag == "failureaction" and element.text == "nonfatal":
raw = etree.Element("{%s}failure_action" %
(RSpec.Namespaces.EMULAB.name))
raw.attrib["action"] = "nonfatal"
node.addRawElement(raw)
pass
pass
nodes[vname] = node
rspec.addResource(node)
pass
if child.tag == "virt_lan_lans":
row = child.find("row")
vname = row.find("vname").text
lan = RSpec.LAN(vname);
lans[vname] = lan;
rspec.addResource(lan)
pass
pass
#
# Now we can do the virt_lans, with the links and interfaces we created
# above.
#
for child in tree.getroot():
if child.tag == "virt_lans":
row = child.find("row")
vname = row.find("vname").text
member = row.find("member").text
lan = lans[vname]
iface = ifaces[member]
mask = row.find("member").text
lan.addInterface(iface)
#
# A lot of these things are per virt_lan, but they are really
# for the entire lan.
#
mask = row.find("member").text
iface.netmask = mask;
if row.find("trivial_ok") != None:
trivial_ok = int(row.find("trivial_ok").text)
if trivial_ok:
lan.trivial_ok = True
pass
pass
if row.find("encap_style") != None:
encap_style = row.find("encap_style").text
if encap_style == "vlan":
lan.vlan_tagging = True
pass
pass
if row.find("emulated") != None:
emulated = int(row.find("emulated").text)
if emulated:
lan.link_multiplexing = True;
pass
pass
pass
pass
#
# Other various things that are in the NS file, that we can handle.
#
for child in tree.getroot():
if child.tag == "portal":
row = child.find("row")
for element in row:
if element.tag == "description":
tour.Description(tour.TEXT, element.text)
elif element.tag == "instructions":
tour.Instructions(tour.TEXT, element.text)
pass
pass
rspec.addTour(tour)
pass
if child.tag == "experiments":
row = child.find("row")
for element in row:
if row.find("encap_style") != None:
encap_style = row.find("encap_style").text
if encap_style == "vlan":
for name,lan in lans.iteritems():
lan.vlan_tagging = True
pass
pass
pass
if row.find("multiplex_factor") != None:
factor = int(row.find("multiplex_factor").text)
rspec.setCollocateFactor(factor)
pass
pass
pass
#
# We only do the startup command right now, since there is no
# event mechanism.
#
if child.tag == "virt_programs":
row = child.find("row")
vnode = row.find("vnode").text
vname = row.find("vname").text
cmd = row.find("command").text
if vname == vnode + "_startcmd":
foo = re.match(r"^\((.*) ; /usr/local/etc/emulab.*\)", cmd);
if foo:
parser = HTMLParser.HTMLParser()
cmd = parser.unescape(foo.group(1));
nodes[vnode].addService(RSpec.Execute("sh", cmd))
pass
pass
#
# Watch for desires that specify a shared node.
#
if child.tag == "virt_node_desires":
row = child.find("row")
vname = row.find("vname").text
desire = row.find("desire").text
if desire == "pcshared":
nodes[vname].exclusive = False
pass
pass
pass
pc.printRequestRSpec(rspec)
#!/usr/bin/perl -wT
#
# Copyright (c) 2000-2014 University of Utah and the Flux Group.
# Copyright (c) 2000-2016 University of Utah and the Flux Group.
#
# {{{EMULAB-LICENSE
#
......@@ -40,16 +40,17 @@ use POSIX ":sys_wait_h";
sub usage()
{
print STDOUT
"Usage: parse-ns [options] pid gid eid nsfile\n".
"Usage: parse-ns [options] [pid gid eid] nsfile\n".
"Where options and arguments are those required by parse.tcl\n";
exit(-1);
}
my $optlist = "nqap";
my $optlist = "nqapc";
my $anonmode = 0;
my $passmode = 0;
my $impotent = 0;
my $debug = 0;
my $catout = 0;
#
# Configure variables
......@@ -136,6 +137,9 @@ if (defined($options{"n"})) {
if (defined($options{"p"})) {
$passmode = 1;
}
if (defined($options{"c"})) {
$catout = 1;
}
if (@ARGV != 4 && @ARGV != 1) {
usage();
}
......@@ -342,8 +346,14 @@ if ($?) {
error => ['ns_parse_failed']},
"Parsing failed (error code $exit_status)!");
}
exit(0)
if ($impotent);
if ($catout) {
system("/bin/cat $outfile");
}
if ($impotent) {
unlink($infile);
unlink($outfile);
exit(0);
}
#
# Run the XML converter on the output.
......
......@@ -133,6 +133,9 @@ Simulator instproc init {args} {
$self instvar new_node_config;
array set new_node_config {}
$self node-config
$self set description ""
$self set instructions ""
}
# renaming the simulator instance
......@@ -485,6 +488,8 @@ Simulator instproc run {} {
$self instvar parameter_descriptions
$self instvar simulated
$self instvar nseconfig
$self instvar description
$self instvar instructions
var_import ::GLOBALS::pid
var_import ::GLOBALS::eid
var_import ::GLOBALS::errors
......@@ -662,6 +667,10 @@ Simulator instproc run {} {
$self spitxml_init
if { $description != "" || $instructions != "" } {
$self spitxml_data "portal" [list "description" "instructions" ] [list $description $instructions ]
}
# update the global nseconfigs using a bogus vname
# i.e. instead of the node on which nse is gonna run
# which was the original vname field, we just put $ns
......@@ -2159,6 +2168,21 @@ Simulator instproc make-simulated {args} {
set simulated 0
}
#
# Portal Stuff
#
Simulator instproc description {text} {
$self instvar description
set description $text
}
Simulator instproc instructions {text} {
$self instvar instructions
set instructions $text
}
#
# Spit out XML
#
......
......@@ -811,34 +811,32 @@ proc tb-proc-tarfiles {cmd args0} { ; # args has special meaning that we
return
}
# Skip the rest in passmode.
if {${GLOBALS::anonymous} || ${GLOBALS::passmode}} {
return
}
# Check the tar file to make sure it exists, is readable, etc...
if {[string match "*://*" $tarfile]} {
# It is a URL, check for a valid protocol.
if {![::TBCOMPAT::verify-url $tarfile]} {
perror "\[$cmd] '$tarfile' is not an http, https, or ftp URL."
# Skip verification in passmode.
if { !${GLOBALS::anonymous} && !${GLOBALS::passmode}} {
# Check the tar file to make sure it exists, is readable, etc...
if {[string match "*://*" $tarfile]} {
# It is a URL, check for a valid protocol.
if {![::TBCOMPAT::verify-url $tarfile]} {
perror "\[$cmd] '$tarfile' is not an http, https, or ftp URL."
return
}
} elseif {![string match "${::TBCOMPAT::PROJROOT}/*" $tarfile] &&
![string match "${::TBCOMPAT::GROUPROOT}/*" $tarfile] &&
![string match "${::TBCOMPAT::USERROOT}/*" $tarfile] &&
(${::TBCOMPAT::SCRATCHROOT} == "" ||
![string match "${::TBCOMPAT::SCRATCHROOT}/*" $tarfile])} {
perror "\[$cmd] '$tarfile' is not in an allowed directory"
return
} elseif {![file exists $tarfile]} {
perror "\[$cmd] '$tarfile' does not exist."
return
} elseif {![file isfile $tarfile]} {
perror "\[$cmd] '$tarfile' is not a file."
return
} elseif {![file readable $tarfile]} {
perror "\[$cmd] '$tarfile' is not readable."
return
}
} elseif {![string match "${::TBCOMPAT::PROJROOT}/*" $tarfile] &&
![string match "${::TBCOMPAT::GROUPROOT}/*" $tarfile] &&
![string match "${::TBCOMPAT::USERROOT}/*" $tarfile] &&
(${::TBCOMPAT::SCRATCHROOT} == "" ||
![string match "${::TBCOMPAT::SCRATCHROOT}/*" $tarfile])} {
perror "\[$cmd] '$tarfile' is not in an allowed directory"
return
} elseif {![file exists $tarfile]} {
perror "\[$cmd] '$tarfile' does not exist."
return
} elseif {![file isfile $tarfile]} {
perror "\[$cmd] '$tarfile' is not a file."
return
} elseif {![file readable $tarfile]} {
perror "\[$cmd] '$tarfile' is not readable."
return
}
# Make sure the tarfile has a valid extension.
......
......@@ -45,6 +45,9 @@ function (_, sup, filesize, JacksEditor, ShowImagingModal, moment, aptforms,
var shareTemplate = _.template(shareString);
var stepsInitialized = false;
var pythonRe = /^import/m;
var tclRe = /^source tb_compat/m;
function initialize()
{
window.APT_OPTIONS.initialize(sup);
......@@ -203,8 +206,7 @@ function (_, sup, filesize, JacksEditor, ShowImagingModal, moment, aptforms,
// Handler for all paths to rspec change (file upload, jacks, edit).
function changeRspec(newRspec)
{
var myRe = /^import/m;
if (myRe.test(newRspec)) {
if (pythonRe.test(newRspec) || tclRe.test(newRspec)) {
//
// A geni-lib script. We are going to pass the script to
// the server to be "run", which returns XML.
......@@ -284,10 +286,12 @@ function (_, sup, filesize, JacksEditor, ShowImagingModal, moment, aptforms,
$('#modal_profile_rspec_textarea').prop("readonly");
// Need to determine the mode.
var myRe = /^import/m;
if (myRe.test(source)) {
if (pythonRe.test(source)) {
mode = "text/x-python";
}
else if (tclRe.test(source)) {
mode = "text/x-tcl";
}
// In case we got here via the modal upload button, need to
// kill the current contents.
$('.CodeMirror').remove();
......@@ -314,10 +318,12 @@ function (_, sup, filesize, JacksEditor, ShowImagingModal, moment, aptforms,
setTimeout(function() {
var source = myCodeMirror.getValue();
var myRe = /^import/m;
if (myRe.test(source)) {
if (pythonRe.test(source)) {
myCodeMirror.setOption("mode", "text/x-python");
}
else if (tclRe.test(source)) {
myCodeMirror.setOption("mode", "text/x-tcl");
}
}, 500);
});
});
......
......@@ -27,6 +27,8 @@ function (_, sup, moment,
var showTemplate = _.template(showString);
var InstTemplate = _.template(instantiateString);
var shareTemplate = _.template(shareString);
var pythonRe = /^import/m;
var tclRe = /^source tb_compat/m;
function initialize()
{
......@@ -123,10 +125,12 @@ function (_, sup, moment,
var mode = "text/xml";
// Need to determine the mode.
var myRe = /^import/m;
if (myRe.test(source)) {
if (pythonRe.test(source)) {
mode = "text/x-python";
}
else if (tclRe.test(source)) {
mode = "text/x-tcl";
}
myCodeMirror = CodeMirror(function(elt) {
$('#modal_profile_rspec_div').prepend(elt);
}, {
......
......@@ -334,12 +334,20 @@ function Do_CheckScript()
SPITAJAX_ERROR(1, "Missing script");
return;
}
if (preg_match("/^import/m", $ajax_args["script"])) {
$command = "webrungenilib";
$warningsfatal = "";
if (isset($ajax_args["warningsfatal"]) && $ajax_args["warningsfatal"]) {
$warningsfatal = "-W";
if (isset($ajax_args["warningsfatal"]) && $ajax_args["warningsfatal"]) {
$command .= " -W";
}
}
elseif (preg_match("/^source tb_compat/m", $ajax_args["script"])) {
$command = "webns2rspec";